{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] 系统找不到指定的路径。: '../input/nvidiaapex/repository/NVIDIA-apex-39e153a'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-085f29e429c2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"../input/nvidiaapex/repository/NVIDIA-apex-39e153a\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m \u001b[1;31m#print(os.listdir(\"../input/glove-global-vectors-for-word-representation\"))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;31m#print(os.listdir(\"../input/jigsaw-unintended-bias-in-toxicity-classification\"))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] 系统找不到指定的路径。: '../input/nvidiaapex/repository/NVIDIA-apex-39e153a'"
     ]
    }
   ],
   "source": [
    "# Version 2 + Bug fix - thanks to @chinhuic\n",
    "\n",
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input/nvidiaapex/repository/NVIDIA-apex-39e153a\"))\n",
    "#print(os.listdir(\"../input/glove-global-vectors-for-word-representation\"))\n",
    "#print(os.listdir(\"../input/jigsaw-unintended-bias-in-toxicity-classification\"))\n",
    "#print(os.listdir(\"../input/fasttext-crawl-300d-2m\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pip/_internal/commands/install.py:207: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\r\n",
      "  cmdoptions.check_install_build_global(options)\r\n",
      "Created temporary directory: /tmp/pip-ephem-wheel-cache-6rk4e96z\r\n",
      "Created temporary directory: /tmp/pip-req-tracker-6u3m77v8\r\n",
      "Created requirements tracker '/tmp/pip-req-tracker-6u3m77v8'\r\n",
      "Created temporary directory: /tmp/pip-install-g1cigjgq\r\n",
      "Processing /kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "  Created temporary directory: /tmp/pip-req-build-ua96mlpg\r\n",
      "  Added file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a to build tracker '/tmp/pip-req-tracker-6u3m77v8'\r\n",
      "    Running setup.py (path:/tmp/pip-req-build-ua96mlpg/setup.py) egg_info for package from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "    Running command python setup.py egg_info\r\n",
      "    torch.__version__  =  1.0.1.post2\r\n",
      "    running egg_info\r\n",
      "    creating pip-egg-info/apex.egg-info\r\n",
      "    writing pip-egg-info/apex.egg-info/PKG-INFO\r\n",
      "    writing dependency_links to pip-egg-info/apex.egg-info/dependency_links.txt\r\n",
      "    writing top-level names to pip-egg-info/apex.egg-info/top_level.txt\r\n",
      "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "    reading manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "  Source in /tmp/pip-req-build-ua96mlpg has version 0.1, which satisfies requirement apex==0.1 from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "  Removed apex==0.1 from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a from build tracker '/tmp/pip-req-tracker-6u3m77v8'\r\n",
      "Installing collected packages: apex\r\n",
      "  Created temporary directory: /tmp/pip-record-8a4z49mu\r\n",
      "  Running setup.py install for apex ... \u001b[?25l    Running command /opt/conda/bin/python -u -c \"import setuptools, tokenize;__file__='/tmp/pip-req-build-ua96mlpg/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" --cpp_ext --cuda_ext install --record /tmp/pip-record-8a4z49mu/install-record.txt --single-version-externally-managed --compile\r\n",
      "    torch.__version__  =  1.0.1.post2\r\n",
      "\r\n",
      "    Compiling cuda extensions with\r\n",
      "    nvcc: NVIDIA (R) Cuda compiler driver\r\n",
      "    Copyright (c) 2005-2018 NVIDIA Corporation\r\n",
      "    Built on Sat_Aug_25_21:08:01_CDT_2018\r\n",
      "    Cuda compilation tools, release 10.0, V10.0.130\r\n",
      "    from /usr/local/cuda/bin\r\n",
      "\r\n",
      "    Pytorch binaries were compiled with Cuda 10.0.130\r\n",
      "\r\n",
      "    running install\r\n",
      "    running build\r\n",
      "    running build_py\r\n",
      "    creating build\r\n",
      "    creating build/lib.linux-x86_64-3.6\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex\r\n",
      "    copying apex/__init__.py -> build/lib.linux-x86_64-3.6/apex\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/__init__.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/LARC.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/multiproc.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/distributed.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/optimized_sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/__init__.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/reparameterization.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/weight_norm.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/handle.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/opt.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_initialize.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/wrap.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/utils.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/rnn_compat.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_amp_state.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/amp.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/frontend.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/compat.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/scaler.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/__version__.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_process_optimizer.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    copying apex/multi_tensor_apply/__init__.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/__init__.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/fp16util.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/loss_scaler.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    copying apex/normalization/__init__.py -> build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    copying apex/normalization/fused_layer_norm.py -> build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/__init__.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/cells.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/RNNBackend.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/models.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/torch_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/functional_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/tensor_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    running build_ext\r\n",
      "    building 'apex_C' extension\r\n",
      "    creating build/temp.linux-x86_64-3.6\r\n",
      "    creating build/temp.linux-x86_64-3.6/csrc\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/opt/conda/include/python3.6m -c csrc/flatten_unflatten.cpp -o build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=apex_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -o build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'amp_C' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/amp_C_frontend.cpp -o build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_scale_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_axpby_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_l2norm_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'fused_adam_cuda' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/fused_adam_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/fused_adam_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'syncbn' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/syncbn.cpp -o build/temp.linux-x86_64-3.6/csrc/syncbn.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/welford.cu -o build/temp.linux-x86_64-3.6/csrc/welford.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/syncbn.o build/temp.linux-x86_64-3.6/csrc/welford.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'fused_layer_norm_cuda' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/layer_norm_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/lib/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/layer_norm_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -maxrregcount=50 -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    running install_lib\r\n",
      "    copying build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/LARC.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/multiproc.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/distributed.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm_kernel.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm_kernel.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fp16_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_adam.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/reparameterization.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/weight_norm.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/handle.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/torch_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/functional_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/tensor_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/opt.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_initialize.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/wrap.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/utils.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/rnn_compat.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_amp_state.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/amp.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/frontend.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/compat.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/scaler.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/__version__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_process_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/multi_tensor_apply.py -> /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16util.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/loss_scaler.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/normalization/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/normalization/fused_layer_norm.py -> /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/cells.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/RNNBackend.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/models.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/LARC.py to LARC.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/distributed.py to distributed.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/handle.py to handle.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/opt.py to opt.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_initialize.py to _initialize.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/wrap.py to wrap.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/utils.py to utils.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/amp.py to amp.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/frontend.py to frontend.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/compat.py to compat.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/scaler.py to scaler.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/__version__.py to __version__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/normalization/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/cells.py to cells.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/models.py to models.cpython-36.pyc\r\n",
      "    running install_egg_info\r\n",
      "    running egg_info\r\n",
      "    creating apex.egg-info\r\n",
      "    writing apex.egg-info/PKG-INFO\r\n",
      "    writing dependency_links to apex.egg-info/dependency_links.txt\r\n",
      "    writing top-level names to apex.egg-info/top_level.txt\r\n",
      "    writing manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    reading manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    writing manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    Copying apex.egg-info to /opt/conda/lib/python3.6/site-packages/apex-0.1-py3.6.egg-info\r\n",
      "    running install_scripts\r\n",
      "    writing list of installed files to '/tmp/pip-record-8a4z49mu/install-record.txt'\r\n",
      "done\r\n",
      "\u001b[?25h  Removing source in /tmp/pip-req-build-ua96mlpg\r\n",
      "Successfully installed apex-0.1\r\n",
      "Cleaning up...\r\n",
      "Removed build tracker '/tmp/pip-req-tracker-6u3m77v8'\r\n",
      "1 location(s) to search for versions of pip:\r\n",
      "* https://pypi.org/simple/pip/\r\n",
      "Getting page https://pypi.org/simple/pip/\r\n",
      "Starting new HTTPS connection (1): pypi.org:443\r\n",
      "Could not fetch URL https://pypi.org/simple/pip/: connection error: HTTPSConnectionPool(host='pypi.org', port=443): Max retries exceeded with url: /simple/pip/ (Caused by NewConnectionError('<pip._vendor.urllib3.connection.VerifiedHTTPSConnection object at 0x7fa690a9a8d0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)) - skipping\r\n"
     ]
    }
   ],
   "source": [
    "# Installing Nvidia Apex\n",
    "! pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/nvidiaapex/repository/NVIDIA-apex-39e153a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'seaborn'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-1df32cf4a704>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpkg_resources\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mscipy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstats\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mstats\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'seaborn'"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import datetime\n",
    "import pkg_resources\n",
    "import seaborn as sns\n",
    "import time\n",
    "import scipy.stats as stats\n",
    "import gc\n",
    "import re\n",
    "import operator \n",
    "import sys\n",
    "from sklearn import metrics\n",
    "from sklearn import model_selection\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torch.nn.functional as F\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import roc_auc_score\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "import os\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "import warnings\n",
    "warnings.filterwarnings(action='once')\n",
    "import pickle\n",
    "from apex import amp\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "device=torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "EPOCHS = 1\n",
    "Data_dir=\"../input/jigsaw-unintended-bias-in-toxicity-classification\"\n",
    "Input_dir = \"../input\"\n",
    "WORK_DIR = \"../working/\"\n",
    "num_to_load=1000000                         #Train size to match time limit\n",
    "valid_size= 100000                          #Validation Size\n",
    "TOXICITY_COLUMN = 'target'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 216, got 192\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# Add the Bart Pytorch repo to the PATH\n",
    "# using files from: https://github.com/huggingface/pytorch-pretrained-BERT\n",
    "package_dir_a = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\n",
    "sys.path.insert(0, package_dir_a)\n",
    "\n",
    "from pytorch_pretrained_bert import convert_tf_checkpoint_to_pytorch\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification,BertAdam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building PyTorch model from configuration: {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "Converting TensorFlow checkpoint from /kaggle/input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/bert_model.ckpt\n",
      "Loading TF weight bert/embeddings/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/embeddings/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/embeddings/position_embeddings with shape [512, 768]\n",
      "Loading TF weight bert/embeddings/token_type_embeddings with shape [2, 768]\n",
      "Loading TF weight bert/embeddings/word_embeddings with shape [30522, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/pooler/dense/bias with shape [768]\n",
      "Loading TF weight bert/pooler/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/predictions/output_bias with shape [30522]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/beta with shape [768]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/bias with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/seq_relationship/output_bias with shape [2]\n",
      "Loading TF weight cls/seq_relationship/output_weights with shape [2, 768]\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'position_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'token_type_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'word_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_weights']\n",
      "Save PyTorch model to ../working/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'../working/bert_config.json'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Translate model from tensorflow to pytorch\n",
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'\n",
    "convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(\n",
    "    BERT_MODEL_PATH + 'bert_model.ckpt',\n",
    "BERT_MODEL_PATH + 'bert_config.json',\n",
    "WORK_DIR + 'pytorch_model.bin')\n",
    "\n",
    "shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__output__.json',\n",
       " 'pytorch_model.bin',\n",
       " '__notebook__.ipynb',\n",
       " 'bert_config.json']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(\"../working\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is the Bert configuration file\n",
    "from pytorch_pretrained_bert import BertConfig\n",
    "\n",
    "bert_config = BertConfig('../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'+'bert_config.json')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting the lines to BERT format\n",
    "# Thanks to https://www.kaggle.com/httpwwwfszyc/bert-in-keras-taming\n",
    "def convert_lines(example, max_seq_length,tokenizer):\n",
    "    max_seq_length -=2\n",
    "    all_tokens = []\n",
    "    longer = 0\n",
    "    for text in tqdm_notebook(example):\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        if len(tokens_a)>max_seq_length:\n",
    "            tokens_a = tokens_a[:max_seq_length]\n",
    "            longer += 1\n",
    "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "        all_tokens.append(one_token)\n",
    "    print(longer)\n",
    "    return np.array(all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded 1100000 records\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2ba625aac664471a4ae9c7cd0566c02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1100000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)\n",
    "train_df = pd.read_csv(os.path.join(Data_dir,\"train.csv\")).sample(num_to_load+valid_size,random_state=SEED)\n",
    "print('loaded %d records' % len(train_df))\n",
    "\n",
    "# Make sure all comment_text values are strings\n",
    "train_df['comment_text'] = train_df['comment_text'].astype(str) \n",
    "\n",
    "sequences = convert_lines(train_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"),MAX_SEQUENCE_LENGTH,tokenizer)\n",
    "train_df=train_df.fillna(0)\n",
    "# List all identities\n",
    "identity_columns = [\n",
    "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
    "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
    "y_columns=['target']\n",
    "\n",
    "train_df = train_df.drop(['comment_text'],axis=1)\n",
    "# convert target to 0,1\n",
    "train_df['target']=(train_df['target']>=0.5).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = sequences[:num_to_load]                \n",
    "y = train_df[y_columns].values[:num_to_load]\n",
    "X_val = sequences[num_to_load:]                \n",
    "y_val = train_df[y_columns].values[num_to_load:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df=train_df.tail(valid_size).copy()\n",
    "train_df=train_df.head(num_to_load)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "train_dataset = torch.utils.data.TensorDataset(torch.tensor(X,dtype=torch.long), torch.tensor(y,dtype=torch.float))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f7e87476dd0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be17e78f91a34fe691b3dccfb76a6f75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abff8445facf495d9168b06d8f313ca6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=31250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output_model_file = \"bert_pytorch.bin\"\n",
    "\n",
    "lr=2e-5\n",
    "batch_size = 32\n",
    "accumulation_steps=2\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(\"../working\",cache_dir=None,num_labels=len(y_columns))\n",
    "model.zero_grad()\n",
    "model = model.to(device)\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "train = train_dataset\n",
    "\n",
    "num_train_optimization_steps = int(EPOCHS*len(train)/batch_size/accumulation_steps)\n",
    "\n",
    "optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                     lr=lr,\n",
    "                     warmup=0.05,\n",
    "                     t_total=num_train_optimization_steps)\n",
    "\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\",verbosity=0)\n",
    "model=model.train()\n",
    "\n",
    "tq = tqdm_notebook(range(EPOCHS))\n",
    "for epoch in tq:\n",
    "    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "    avg_loss = 0.\n",
    "    avg_accuracy = 0.\n",
    "    lossf=None\n",
    "    tk0 = tqdm_notebook(enumerate(train_loader),total=len(train_loader),leave=False)\n",
    "    optimizer.zero_grad()   # Bug fix - thanks to @chinhuic\n",
    "    for i,(x_batch, y_batch) in tk0:\n",
    "#        optimizer.zero_grad()\n",
    "        y_pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)\n",
    "        loss =  F.binary_cross_entropy_with_logits(y_pred,y_batch.to(device))\n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "        if (i+1) % accumulation_steps == 0:             # Wait for several backward steps\n",
    "            optimizer.step()                            # Now we can do an optimizer step\n",
    "            optimizer.zero_grad()\n",
    "        if lossf:\n",
    "            lossf = 0.98*lossf+0.02*loss.item()\n",
    "        else:\n",
    "            lossf = loss.item()\n",
    "        tk0.set_postfix(loss = lossf)\n",
    "        avg_loss += loss.item() / len(train_loader)\n",
    "        avg_accuracy += torch.mean(((torch.sigmoid(y_pred[:,0])>0.5) == (y_batch[:,0]>0.5).to(device)).to(torch.float) ).item()/len(train_loader)\n",
    "    tq.set_postfix(avg_loss=avg_loss,avg_accuracy=avg_accuracy)\n",
    "\n",
    "\n",
    "torch.save(model.state_dict(), output_model_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2834b4dadbde4a1f94ee43d7f22d2fcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3125), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Run validation\n",
    "# The following 2 lines are not needed but show how to download the model for prediction\n",
    "model = BertForSequenceClassification(bert_config,num_labels=len(y_columns))\n",
    "model.load_state_dict(torch.load(output_model_file ))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad=False\n",
    "model.eval()\n",
    "valid_preds = np.zeros((len(X_val)))\n",
    "valid = torch.utils.data.TensorDataset(torch.tensor(X_val,dtype=torch.long))\n",
    "valid_loader = torch.utils.data.DataLoader(valid, batch_size=32, shuffle=False)\n",
    "\n",
    "tk0 = tqdm_notebook(valid_loader)\n",
    "for i,(x_batch,)  in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)\n",
    "    valid_preds[i*32:(i+1)*32]=pred[:,0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From baseline kernel\n",
    "\n",
    "def calculate_overall_auc(df, model_name):\n",
    "    true_labels = df[TOXICITY_COLUMN]>0.5\n",
    "    predicted_labels = df[model_name]\n",
    "    return metrics.roc_auc_score(true_labels, predicted_labels)\n",
    "\n",
    "def power_mean(series, p):\n",
    "    total = sum(np.power(series, p))\n",
    "    return np.power(total / len(series), 1 / p)\n",
    "\n",
    "def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):\n",
    "    bias_score = np.average([\n",
    "        power_mean(bias_df[SUBGROUP_AUC], POWER),\n",
    "        power_mean(bias_df[BPSN_AUC], POWER),\n",
    "        power_mean(bias_df[BNSP_AUC], POWER)\n",
    "    ])\n",
    "    return (OVERALL_MODEL_WEIGHT * overall_auc) + ((1 - OVERALL_MODEL_WEIGHT) * bias_score)\n",
    "\n",
    "\n",
    "\n",
    "SUBGROUP_AUC = 'subgroup_auc'\n",
    "BPSN_AUC = 'bpsn_auc'  # stands for background positive, subgroup negative\n",
    "BNSP_AUC = 'bnsp_auc'  # stands for background negative, subgroup positive\n",
    "\n",
    "def compute_auc(y_true, y_pred):\n",
    "    try:\n",
    "        return metrics.roc_auc_score(y_true, y_pred)\n",
    "    except ValueError:\n",
    "        return np.nan\n",
    "\n",
    "def compute_subgroup_auc(df, subgroup, label, model_name):\n",
    "    subgroup_examples = df[df[subgroup]>0.5]\n",
    "    return compute_auc((subgroup_examples[label]>0.5), subgroup_examples[model_name])\n",
    "\n",
    "def compute_bpsn_auc(df, subgroup, label, model_name):\n",
    "    \"\"\"Computes the AUC of the within-subgroup negative examples and the background positive examples.\"\"\"\n",
    "    subgroup_negative_examples = df[(df[subgroup]>0.5) & (df[label]<=0.5)]\n",
    "    non_subgroup_positive_examples = df[(df[subgroup]<=0.5) & (df[label]>0.5)]\n",
    "    examples = subgroup_negative_examples.append(non_subgroup_positive_examples)\n",
    "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
    "\n",
    "def compute_bnsp_auc(df, subgroup, label, model_name):\n",
    "    \"\"\"Computes the AUC of the within-subgroup positive examples and the background negative examples.\"\"\"\n",
    "    subgroup_positive_examples = df[(df[subgroup]>0.5) & (df[label]>0.5)]\n",
    "    non_subgroup_negative_examples = df[(df[subgroup]<=0.5) & (df[label]<=0.5)]\n",
    "    examples = subgroup_positive_examples.append(non_subgroup_negative_examples)\n",
    "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
    "\n",
    "def compute_bias_metrics_for_model(dataset,\n",
    "                                   subgroups,\n",
    "                                   model,\n",
    "                                   label_col,\n",
    "                                   include_asegs=False):\n",
    "    \"\"\"Computes per-subgroup metrics for all subgroups and one model.\"\"\"\n",
    "    records = []\n",
    "    for subgroup in subgroups:\n",
    "        record = {\n",
    "            'subgroup': subgroup,\n",
    "            'subgroup_size': len(dataset[dataset[subgroup]>0.5])\n",
    "        }\n",
    "        record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)\n",
    "        record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)\n",
    "        record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)\n",
    "        records.append(record)\n",
    "    return pd.DataFrame(records).sort_values('subgroup_auc', ascending=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bnsp_auc</th>\n",
       "      <th>bpsn_auc</th>\n",
       "      <th>subgroup</th>\n",
       "      <th>subgroup_auc</th>\n",
       "      <th>subgroup_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.970580</td>\n",
       "      <td>0.850340</td>\n",
       "      <td>homosexual_gay_or_lesbian</td>\n",
       "      <td>0.842154</td>\n",
       "      <td>579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.972660</td>\n",
       "      <td>0.853721</td>\n",
       "      <td>white</td>\n",
       "      <td>0.848294</td>\n",
       "      <td>1312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.978164</td>\n",
       "      <td>0.829833</td>\n",
       "      <td>black</td>\n",
       "      <td>0.855691</td>\n",
       "      <td>744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.965812</td>\n",
       "      <td>0.893783</td>\n",
       "      <td>muslim</td>\n",
       "      <td>0.872753</td>\n",
       "      <td>1076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.957125</td>\n",
       "      <td>0.921044</td>\n",
       "      <td>jewish</td>\n",
       "      <td>0.892779</td>\n",
       "      <td>387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.960471</td>\n",
       "      <td>0.944279</td>\n",
       "      <td>female</td>\n",
       "      <td>0.925204</td>\n",
       "      <td>2790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.951274</td>\n",
       "      <td>0.955246</td>\n",
       "      <td>christian</td>\n",
       "      <td>0.926057</td>\n",
       "      <td>1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.968627</td>\n",
       "      <td>0.931659</td>\n",
       "      <td>male</td>\n",
       "      <td>0.928642</td>\n",
       "      <td>2260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.981058</td>\n",
       "      <td>0.929391</td>\n",
       "      <td>psychiatric_or_mental_illness</td>\n",
       "      <td>0.955801</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bnsp_auc  bpsn_auc      ...       subgroup_auc  subgroup_size\n",
       "2  0.970580  0.850340      ...           0.842154            579\n",
       "7  0.972660  0.853721      ...           0.848294           1312\n",
       "6  0.978164  0.829833      ...           0.855691            744\n",
       "5  0.965812  0.893783      ...           0.872753           1076\n",
       "4  0.957125  0.921044      ...           0.892779            387\n",
       "1  0.960471  0.944279      ...           0.925204           2790\n",
       "3  0.951274  0.955246      ...           0.926057           1997\n",
       "0  0.968627  0.931659      ...           0.928642           2260\n",
       "8  0.981058  0.929391      ...           0.955801            227\n",
       "\n",
       "[9 rows x 5 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.9298784415544122"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "MODEL_NAME = 'model1'\n",
    "test_df[MODEL_NAME]=torch.sigmoid(torch.tensor(valid_preds)).numpy()\n",
    "TOXICITY_COLUMN = 'target'\n",
    "bias_metrics_df = compute_bias_metrics_for_model(test_df, identity_columns, MODEL_NAME, 'target')\n",
    "bias_metrics_df\n",
    "get_final_metric(bias_metrics_df, calculate_overall_auc(test_df, MODEL_NAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
